


import requests
from lxml import etree



class XiaoshuoSpider():
    def __init__(self, url):
        self.parse(url)

    def set_lxml(self, text):
        return etree.HTML(text)

    def Request(self, url):
        response = requests.get(url=url)
        return self.set_lxml(response.text)

    def parse(self, url):
        """ 第一级页面 """
        # print(response.body)
        response = self.Request(url)

        novellist = response.xpath('//div[@id="main"]/div[@class="novellist"]/ul/li/a/@href')
        for href in novellist:
            print(href)
            url = href.get()
            res_two = self.parse_two(url)

    def parse_two(self, url):
        """ 章节url """
        response = self.Request(url)


        chapter_list = response.xpath('//div[@id="list"]/dl/dd/a')
        for chapter in chapter_list:
            chapter_href = chapter.xpath('./@href').get()
            chapter_name = chapter.xpath('./text()').get()
            url = 'http://www.xbiquge.la' + chapter_href
            # print('第二级页面链接： ',url)
            self.parse_three(url)

    def parse_three(self, url):
        """ 章节内容 """
        response = self.Request(url)
        content = response.xpath('//div[@id="content"]/text()').getall()

        content = '\n'.join(content)
        content = content.replace('\r','').replace('\xa0','')


        head_detail = response.xpath('//meta[@name="keywords"]/@content').get()

        name = head_detail.split(',')[0] #小说名称
        chapter_name = head_detail.split(',')[-1] # 章节名称

        print('name:{}\nchapter_name:{}\ncontent:{}'.format(name, chapter_name, content))

if __name__ == '__main__':

    start_url = 'http://www.xbiquge.la/xiaoshuodaquan/'
    XiaoshuoSpider(start_url)